import os
import scrapy
from sdWeatherSpider.items import SdweatherspiderItem

class EverycityinsdSpider(scrapy.Spider):
    name = 'everyCityinSD'
    allowed_domains = ['www.weather.com.cn']
    start_urls = ['http://www.weather.com.cn/shandong/index.shtml']

    def start_requests(self):
        # 检查并删除 weather.txt 文件（如果存在）
        file_path = 'weather.txt'
        if os.path.exists(file_path):
            try:
                os.remove(file_path)
                self.logger.info(f"Successfully removed {file_path}")
            except Exception as e:
                self.logger.error(f"Failed to remove {file_path}: {e}")
        else:
            self.logger.info(f"{file_path} does not exist, skipping removal.")

        # 开始请求页面
        for url in self.start_urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        # 获取每个地市的链接地址
        urls = response.css('dt>a[title]::attr(href)').getall()
        for url in urls:
            # 针对每个链接地址发起请求
            yield scrapy.Request(url=url, callback=self.parse_city)

    def parse_city(self, response):
        '''处理每个地市天气预报链接地址的实例方法'''
        # 用来存储采集到的信息的对象
        item = SdweatherspiderItem()

        # 获取城市名称
        city = response.xpath('//div[@class="crumbs fl"]/a[3]/text()').get()
        item['city'] = city

        # 定位包含天气预报信息的ul节点，其中每个li节点存放一天的天气
        selectors = response.xpath('//ul[@class="t clearfix"]')

        weather = []
        # 遍历当前ul节点中的所有li节点，提取每天的天气信息
        for li in selectors.xpath('./li'):
            # 提取日期
            date = li.xpath('./h1/text()').get()
            # 云的情况
            cloud = li.xpath('./p[@title]/text()').get()
            # 晚上页面中不显示今天的高温，返回字符串'none'
            high = li.xpath('./p[@class="tem"]/span/text()').get('none')
            low = li.xpath('./p[@class="tem"]/i/text()').get()
            wind = li.xpath('./p[@class="win"]/em/span[1]/@title').get()
            wind += ',' + li.xpath('./p[@class="win"]/i/text()').get()
            weather.append(f'{date}:{cloud},{high}/{low},{wind}')

        item['weather'] = '\n'.join(weather)
        yield item  # 使用yield返回结果
